package main

import (
	"bufio"
	"fmt"
	"log"
	"os"

	//"github.com/gocolly/colly"

	"github.com/gocolly/colly/v2"
)

var UsedLink map[string]bool

func main() {
	UsedLink = make(map[string]bool)
	// Instantiate default collector
	c := colly.NewCollector(
		// Visit only domains: hackerspaces.org, wiki.hackerspaces.org
		colly.AllowedDomains("xiaodiaodaya.cn"),
	)

	// On every a element which has href attribute call callback
	c.OnHTML("div", func(e *colly.HTMLElement) {
		e.ForEach("a", func(i int, element *colly.HTMLElement) {
			link := element.Attr("href")
			if IfAccLink(e.Request.AbsoluteURL(link)) {
				AddLink(e.Request.AbsoluteURL(link))
				c.Visit(e.Request.AbsoluteURL(link))
			}
		})
	})
	//c.OnHTML("div[class=\"hotword_list\"]", func(e *colly.HTMLElement) {
	//	e.ForEach("a", func(i int, element *colly.HTMLElement) {
	//		link := element.Attr("href")
	//		if IfAccLink(e.Request.AbsoluteURL(link)) {
	//			AddLink(e.Request.AbsoluteURL(link))
	//			c.Visit(e.Request.AbsoluteURL(link))
	//		}
	//	})
	//})
	//c.OnHTML("")
	c.OnHTML(".content", func(ele *colly.HTMLElement) {
		text := ele.ChildText("h2[class=\"titleview\"]")
		fmt.Println("______________", text)
		//fmt.Println(ele.Text)
		file, err := os.OpenFile("./file/"+text+".txt", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			log.Println(err)
			return
		}
		writer := bufio.NewWriter(file)
		_, err = writer.WriteString(ele.Text)
		if err != nil {
			panic(err)
		}
		writer.Flush()
		file.Close()
	})
	// Before making a request print "Visiting ..."
	c.OnRequest(func(r *colly.Request) {
		fmt.Println("Visiting", r.URL.String())
	})

	// Start scraping on https://hackerspaces.org
	c.Visit("http://xiaodiaodaya.cn/")
}

func AddLink(link string) (err error) {
	UsedLink[link] = true
	return nil
}

func IfAccLink(link string) bool {
	for key, _ := range UsedLink {
		if key == link {
			return false
		}
		//fmt.Println("__________________--key=", key)

	}
	fmt.Println("_________________--link=", link)

	return true
}
